import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instances;
import edu.gatech.doi.spamframework.model.ModelAttribute;


public class IDSpam {
	private HashSet<String> personspamhs=null;
	private HashSet<String> messagespamhs=null;
	private HashSet<String> messageidhs=null;
	
	public HashSet<String> loadFile2HashSet(String filename)
	{
		HashSet<String> temphs=new HashSet<String>();
		try {
			FileReader frs=new FileReader(filename);
			BufferedReader br=new BufferedReader(frs);
			String line=null;
			while((line=br.readLine())!=null)
			{
				if(!line.equalsIgnoreCase(""))
				temphs.add(line.trim());
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return temphs;
	}
	
	public ArrayList<String> loadFile2ArrayList(String filename)
	{
		ArrayList<String> temphs=new ArrayList<String>();
		try {
			FileReader frs=new FileReader(filename);
			BufferedReader br=new BufferedReader(frs);
			String line=null;
			while((line=br.readLine())!=null)
			{
				if(!line.equalsIgnoreCase(""))
				temphs.add(line.trim());
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return temphs;
	}
	
	
	public void HashSet2File(HashSet<String> hs, String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			
		
			for(String temp:hs)
			{
				br.write(temp);
				br.newLine();
				br.flush();
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public HashSet<String> UIDbySID(HashSet<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashSet<String> result=new HashSet<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 
			
			// if()
			Iterator it=hs.iterator();
			int totalnum=0;
			while(it.hasNext())
			{
				String temps=(String) it.next();
				System.out.println("SELECT user_id FROM status where id="+temps+";");
				stmt.execute("SELECT user_id FROM status where id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					{
					result.add(rs.getString(1));
					System.out.println("user_id:"+rs.getString(1));
					}
				}
				rs.close();
//				try {
//					Thread.sleep(1000);
//				} catch (InterruptedException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				totalnum++;
				System.out.println("totalnum:"+totalnum);
				System.out.println("out~~");
				//rs=null;
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	
	public HashMap<String,Integer> UIDHMbySID(ArrayList<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashMap<String,Integer> result=new HashMap<String,Integer>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 
			
			// if()
			Iterator it=hs.iterator();
			int totalnum=0;
			while(it.hasNext())
			{
				String temps=(String) it.next();
				System.out.println("SELECT user_id FROM status where id="+temps+";");
				stmt.execute("SELECT user_id FROM status where id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					{

						String key=rs.getString(1);
						if(result.containsKey(key))
						{

							Integer value=result.get(key);
							result.put(key,value.intValue()+1);
						}
						else
						{
							result.put(key,1);
						}

						System.out.println("user_id:"+rs.getString(1));
					}
				}
				rs.close();
//				try {
//					Thread.sleep(1000);
//				} catch (InterruptedException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				totalnum++;
				System.out.println("totalnum:"+totalnum);
				System.out.println("out~~");
				//rs=null;
			}

		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;
	}
	
	
	public ArrayList<String> UIDbySID(ArrayList<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		ArrayList<String> result=new ArrayList<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 
			
			// if()
			Iterator it=hs.iterator();
			int totalnum=0;
			while(it.hasNext())
			{
				String temps=(String) it.next();
				System.out.println("SELECT user_id FROM status where id="+temps+";");
				stmt.execute("SELECT user_id FROM status where id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					{
					result.add(rs.getString(1));
					System.out.println("user_id:"+rs.getString(1));
					}
				}
				rs.close();
//				try {
//					Thread.sleep(1000);
//				} catch (InterruptedException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				totalnum++;
				System.out.println("totalnum:"+totalnum);
				System.out.println("out~~");
				//rs=null;
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	

	public HashMap<String,Integer> UIDbySID(HashMap<String,Integer> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashMap<String,Integer> result=new HashMap<String,Integer>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 
			
			// if()
			Iterator it=hs.keySet().iterator();
			int totalnum=0;
			while(it.hasNext())
			{
				String temps=(String) it.next();
				int label=hs.get(temps);
				System.out.println("SELECT user_id FROM status where id="+temps+";");
				stmt.execute("SELECT user_id FROM status where id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					{
						String key=rs.getString(1);
						if(!result.containsKey(key.trim()))
						{
							if(label==1)
								result.put(key.trim(),1);
							else
								result.put(key.trim(), 0);
						}
						else
						{
							int value=result.get(key);
							if(label==1)
								result.put(key.trim(),value+1);
							
						}
						
					System.out.println("user_id:"+rs.getString(1));
					}
				}
				rs.close();
//				try {
//					Thread.sleep(1000);
//				} catch (InterruptedException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				totalnum++;
				System.out.println("totalnum:"+totalnum);
				System.out.println("out~~");
				//rs=null;
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	
	public HashSet<String> SIDbyUID(HashSet<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashSet<String> result=new HashSet<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 

			// if()
			for(String temps:hs)
			{
				stmt.execute("SELECT id FROM status where user_id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					result.add(rs.getString(1));
				}
				rs.close();
				
				
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	
	public HashSet<String> UrlIDbySID(HashSet<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashSet<String> result=new HashSet<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 

			// if()
			for(String temps:hs)
			{
				stmt.execute("SELECT url_id FROM web_content where status_id="+temps+" and file_path not like '%_redirect%'"+";" );

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					result.add(rs.getString(1));
				}
				rs.close();
				
				
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	
	public HashSet<String> SIDbyUrlID(HashSet<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashSet<String> result=new HashSet<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 

			// if()
			for(String temps:hs)
			{
				if(temps!=null&&!temps.equalsIgnoreCase(""))
				{
					stmt.execute("SELECT status_id FROM web_content where url_id="+temps+";" );

					// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				//	System.out.println("start query "); 
					rs = stmt.getResultSet();
				//	System.out.println("done query "); 
					if(rs!=null)
					{
						while(rs.next())
							result.add(rs.getString(1));
					}
					rs.close();
				
				}
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		System.out.println("the size of result is:"+result.size());
		return result;

	}
	
	
	public ArrayList<String> SIDbyUrlID(ArrayList<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		ArrayList<String> result=new ArrayList<String>();
		Connection conn = null;


		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 

			// if()
			for(String temps:hs)
			{
				if(temps!=null&&!temps.equalsIgnoreCase(""))
				{
					stmt.execute("SELECT status_id FROM web_content where url_id="+temps+";" );

					// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
					System.out.println("start query "); 
					rs = stmt.getResultSet();
					System.out.println("done query "); 
					if(rs!=null)
					{
						while(rs.next())
							result.add(rs.getString(1));
					}
					rs.close();
				
				}
			}


		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	

	public HashSet<String> StatsByUID(HashSet<String> hs)
	{
		/**
		 * connect to db :Mysql
		 */

		HashSet<String> result=new HashSet<String>();
		Connection conn = null;

		//statistics
		int maxFriends_count=0;
		int minFriends_count=747974;
		long totalfriendscount=0;
		double avgFriends_count;
		int maxFollowers_count=0;
		int minFollowers_count=4488335;
		long totalfollowerscount=0;
		double avgFollowers_count;
		int maxFavourites_count=0;
		int minFavourites_count=128144;
		long totalfavouritescount=0;
		double avgFavourites_count;
		int maxStatuses_count=0;
		int minStatuses_count=710680;
		long totalstatusescount=0;
		double avgStatuses_count;
		int totalnum=0;
		int nullurlnum=0;
		int bothnullnum=0;
		int descnullnum=0;

		System.out.println("before connecting...");
		try {
			conn =DriverManager.getConnection("jdbc:postgresql://matcha.cc.gatech.edu:5432/twitware"+
					"?user=wang6"+"&password=?WDcj984615?");
		} catch (SQLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("connect successfully");
		// Do something with the Connection
		Statement stmt = null;
		ResultSet rs = null;
		try {
			stmt = (Statement) conn.createStatement();
			//rs = stmt.executeQuery("SELECT foo FROM bar");
			// or alternatively, if you don't know ahead of time that
			// the query will be a SELECT...

			stmt.execute("set search_path to twitware,public;");
			System.out.println("set search_path to twitware,public;");
			stmt.setFetchSize(100); 
			
			// if()
			for(String temps:hs)
			{
				stmt.execute("SELECT url,friends_count,followers_count,favourites_count,statuses_count,description FROM user_info where id="+temps+";");

				// System.out.println("SELECT id,name,location,description,url FROM user_info limit 5000 offset "+offsetnum+";");
				System.out.println("start query "); 
				rs = stmt.getResultSet();
				System.out.println("done query "); 
				if(rs!=null)
				{
					while(rs.next())
					{
						String tempurl=rs.getString(1);
						
						int tempfriends=rs.getInt(2);
						int tempfollowers=rs.getInt(3);
						int tempfavourites=rs.getInt(4);
						int tempstatuses=rs.getInt(5);
						String tempdesc=rs.getString(6);
						
						if(tempurl==null||tempurl.trim().equalsIgnoreCase(""))
						{
							nullurlnum++;
							if(tempdesc==null||tempdesc.trim().equalsIgnoreCase("")||!tempdesc.contains("http:"))
								bothnullnum++;
							
						}
						if(tempdesc==null||tempdesc.trim().equalsIgnoreCase(""))
						{
							descnullnum++;
						}
						
						if(tempfriends>maxFriends_count)
						{
							maxFriends_count=tempfriends;
						}
						if(tempfriends<minFriends_count)
						{
							minFriends_count=tempfriends;
						}
						
						if(tempfollowers>maxFollowers_count)
						{
							maxFollowers_count=tempfollowers;
						}
						if(tempfollowers<minFollowers_count)
						{
							minFollowers_count=tempfollowers;
						}
						
						if(tempfavourites>maxFavourites_count)
						{
							maxFavourites_count=tempfavourites;
						}
						if(tempfavourites<minFavourites_count)
						{
							minFavourites_count=tempfavourites;
						}
						if(tempstatuses>maxStatuses_count)
						{
							maxStatuses_count=tempstatuses;
						}
						if(tempstatuses<minStatuses_count)
						{
							minStatuses_count=tempstatuses;
						}
						totalfriendscount=totalfriendscount+tempfriends;
						totalfollowerscount=totalfollowerscount+tempfollowers;
						totalfavouritescount=totalfavouritescount+tempfavourites;
						totalstatusescount=totalstatusescount+tempstatuses;
						totalnum++;
						
					}
					
					
				}
				rs.close();
				//stmt.close();
				
			}
			rs=null;
			stmt=null;
			avgFriends_count=(double)totalfriendscount/(double)totalnum;
			avgFollowers_count=(double)totalfollowerscount/(double)totalnum;
			avgFavourites_count=(double)totalfavouritescount/(double)totalnum;
			avgStatuses_count=(double)totalstatusescount/(double)totalnum;
			
			System.out.println("TotalNum:"+totalnum);
			System.out.println(" Null URL:"+nullurlnum);
			System.out.println(" Null desc:"+descnullnum);
			System.out.println("both Null desc:"+bothnullnum);
			System.out.println("Friends_count:"+"\n"+
					"Max:"+maxFriends_count+
					"Min:"+minFriends_count+
					"Avg:"+avgFriends_count+"\n"+
					"Followers_count:"+"\n"+
					"Max:"+maxFollowers_count+
					"Min:"+minFollowers_count+
					"Avg:"+avgFollowers_count+"\n"+
					"Favourites_count:"+"\n"+
					"Max:"+maxFavourites_count+
					"Min:"+minFavourites_count+
					"Avg:"+avgFavourites_count+"\n"+
					"Statuese_count:"+"\n"+
					"Max:"+maxStatuses_count+
					"Min:"+minStatuses_count+
					"Avg:"+avgStatuses_count+"\n");

		}catch (SQLException ex){
			// handle any errors
			System.out.println("SQLException: " + ex.getMessage());
			System.out.println("SQLState: " + ex.getSQLState());
			System.out.println("VendorError: " + ex.getErrorCode());
		}
		    
		return result;

	}
	
	
	public void compareID(ArrayList<String> hs1,ArrayList<String> hs2,String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			
		
			for(String temp:hs1)
			{
				if(hs2.contains(temp))
					br.write(temp+" 1");
				else
					br.write(temp+" 0");
				br.newLine();
				br.flush();
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public void compareSID(ArrayList<String> hs1,ArrayList<String> hs2,String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			
		
			for(String temp:hs1)
			{
				if(hs2.contains(temp))
					br.write(temp+" 1");
				else
					br.write(temp+" 0");
				br.newLine();
				br.flush();
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	public void compareID(HashSet<String> hs1,HashSet<String> hs2,String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			
		
			for(String temp:hs1)
			{
				if(hs2.contains(temp))
					br.write(temp+" 1");
				else
					br.write(temp+" 0");
				br.newLine();
				br.flush();
			}
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	
	public void compareUID(ArrayList<String> hs1,HashMap<String,Integer> hs2,String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			

			for(String temp:hs1)
			{
				
				if(hs2.containsKey(temp))
				{
					Integer value=hs2.get(temp);
					int v=value.intValue();
					br.write(temp+" "+v);
				
				}
				else
				{
					br.write(temp+" 0");
				}
				br.newLine();
				br.flush();
			}
			
			
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	
	
	public void compareSID(HashSet<String> hs1,HashSet<String> hs2,String filename)
	{
		try {
			File file=new File(filename);
			if(!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				file.delete();
				file.createNewFile();
			}
			FileWriter frs=new FileWriter(filename);
			BufferedWriter br=new BufferedWriter(frs);
			String line=null;
			
			HashMap<String,Integer> hm=new HashMap<String,Integer>();
			
			for(String temp:hs1)
			{
				hm.put(temp, 0);		
			}
			
			for(String temp:hs1)
			{
				if(hs2.contains(temp))
				{
					Integer value=hm.get(temp);
					int v=value.intValue();
					hm.put(temp, v+1);
				}
			}
			
			Iterator it=hm.keySet().iterator();
			while(it.hasNext())
			{
				String temp=(String) it.next();
				Integer value=hm.get(temp);
				int v=value.intValue();
				if(hs2.contains(temp))
					br.write(temp+" "+v);
				br.newLine();
				br.flush();
			}
			
			
			br.close();
			frs.close();
			
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public static void main(String args[])
	{
		IDSpam ids=new IDSpam();
		String uidfilename=args[0];
		String sidfilename=args[1];
		String filename=args[2];
		HashSet<String> uidhs=ids.loadFile2HashSet(uidfilename);
		HashSet<String> sidhs=ids.loadFile2HashSet(sidfilename);
		
		HashSet<String> sidhs1=ids.SIDbyUrlID(sidhs);
		
//		ids.compareUID(uidhs, sidhs1, filename);
		
//		HashSet<String> sidhs=ids.loadFile2HashSet(sidfilename);
//		HashSet<String> sidhs=ids.SIDbyUID(uidhs);
//		ids.HashSet2File(sidhs, sidfilename);
//		HashSet<String> sidhs=ids.UrlIDbySID(uidhs);
//		ids.HashSet2File(sidhs, sidfilename);
		
		
//		HashSet<String> usidhs=ids.UIDbySID(sidhs);
//		ids.StatsByUID(uidhs);
//		int totalnum=0;
		
		
//		Iterator it=usidhs.iterator();
//		System.out.println(usidhs.size());
//		while(it.hasNext())
//		{
//			String temps=(String) it.next();
//			if(uidhs.contains(temps))
//			{
//				totalnum++;
//				System.out.println("num:"+totalnum);
//			}
//		}
//		System.out.println("totalnum:"+totalnum);
		
	}
}


